package com.mxsoft.shjjw;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.mxsoft.util.HttpClientUtil;

import java.io.FileWriter;
import java.io.IOException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.RecursiveTask;

public class Crawler_shjjw {

    private static String url = "http://www.shjjw.gov.cn/fg/wygl/glqy/s?pn={{pn}}&ps={{ps}}";
    private static String url2 = "http://www.shjjw.gov.cn/fg/wygl/qyxx?id={{cid}}";

    private static String area_url = "http://www.shjjw.gov.cn/fg/wygl/wyxq/s?pn={{pn}}&ps={{ps}}";

    private static String area_detail_url = "http://www.shjjw.gov.cn/fg/wygl/xq/?id={{id}}";

    private static int MAX = 20;

    public static void main(String[] args) throws IOException {
        crawlerCompany();


//        crawlerArea();
    }

    public static void crawlerArea() throws IOException {
        //1\ get data count
        //2\ async get data

        int pn = 1;
        int ps = 1;

        String result = doGetData(area_url, pn, ps);

        JSONObject jsonObject = JSON.parseObject(result);

        if (!jsonObject.getInteger("flag").equals(0)) {
            System.out.println("请求失败" + jsonObject.getString("msg"));
            return;
        }

        JSONObject data = jsonObject.getJSONObject("data");
        Integer total = data.getInteger("total");
        System.out.println("total: "+total);
        if (total < 1) {
            System.out.println("请求数据为空");
            return;
        }

//        System.out.println(data.getJSONArray("list"));
        ForkJoinPool fjp = new ForkJoinPool();
        AreaForkJoinTask companyForkJoinTask = new AreaForkJoinTask(1, total/10+1);
        long startTime = System.currentTimeMillis();
        JSONArray jsonArray = fjp.invoke(companyForkJoinTask);
        long endTime = System.currentTimeMillis();

        System.out.println("evlucate takes " + (endTime - startTime) /1000  + " seconds");

        FileWriter fileWriter = new FileWriter("area.csv");

        for (int i = 0 ; i < jsonArray.size(); i ++) {
            jsonObject = jsonArray.getJSONObject(i);
            if (null == jsonObject) {
                continue;
            }
            fileWriter.write(jsonObject.getString("areamanager") == null ? "" : jsonObject.getString("areamanager"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("committeename") == null ? "" : jsonObject.getString("committeename"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("concretaddr") == null ? "": jsonObject.getString("concretaddr"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("companyname") == null ? "" : jsonObject.getString("companyname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("daytel") == null ? "" : jsonObject.getString("daytel"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("distname") == null ? "" : jsonObject.getString("distname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("duratime") == null ? "" : jsonObject.getString("duratime"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("housekind") == null ? "" : jsonObject.getString("housekind"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("keyid") == null ? "" : jsonObject.getString("keyid"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("linktel") == null ? "" : jsonObject.getString("linktel"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("membername") == null ? "" : jsonObject.getString("membername"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("nighttel") == null ? "" : jsonObject.getString("nighttel"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("officeaddr") == null ? "" : jsonObject.getString("officeaddr"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("ownercongressname") == null ? "" : jsonObject.getString("ownercongressname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("projectname") == null ? "" : jsonObject.getString("projectname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("startdate") == null ? "" : jsonObject.getString("startdate"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("streetname") == null ? "" : jsonObject.getString("streetname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("tel") == null ? "" : jsonObject.getString("tel"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("totalbuildingarea") == null ? "" : jsonObject.getString("totalbuildingarea"));
            fileWriter.write("\t");
            fileWriter.write("\n");


        }
        fileWriter.flush();
        fileWriter.close();
    }

    public static void crawlerCompany() throws IOException {
        //1\ get data count
        //2\ async get data

        int pn = 1;
        int ps = 1;

        String result = doGetData(url, pn, ps);

        JSONObject jsonObject = JSON.parseObject(result);

        if (!jsonObject.getInteger("flag").equals(0)) {
            System.out.println("请求失败" + jsonObject.getString("msg"));
            return;
        }

        JSONObject data = jsonObject.getJSONObject("data");
        Integer total = data.getInteger("total");
        System.out.println("total: "+total);
        if (total < 1) {
            System.out.println("请求数据为空");
            return;
        }

//        System.out.println(data.getJSONArray("list"));
        ForkJoinPool fjp = new ForkJoinPool();
//        fjp.getParallelism();
        CompanyForkJoinTask companyForkJoinTask = new CompanyForkJoinTask(1, total/10+1);
        long startTime = System.currentTimeMillis();
        JSONArray jsonArray = fjp.invoke(companyForkJoinTask);
        long endTime = System.currentTimeMillis();

        System.out.println("evlucate takes " + (endTime - startTime) /1000  + " seconds");

        FileWriter fileWriter = new FileWriter("company.csv");

        for (int i = 0 ; i < jsonArray.size(); i ++) {
            jsonObject = jsonArray.getJSONObject(i);
            if (null == jsonObject) {
                continue;
            }
            fileWriter.write(jsonObject.getString("zip") == null ? "" : jsonObject.getString("zip"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("gradename") == null ? "" : jsonObject.getString("gradename"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("companyid") == null ? "": jsonObject.getString("companyid"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("companyname") == null ? "" : jsonObject.getString("companyname"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("tel") == null ? "" : jsonObject.getString("tel"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("legalrept") == null ? "" : jsonObject.getString("legalrept"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("regaddr") == null ? "" : jsonObject.getString("regaddr"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("releasedate") == null ? "" : jsonObject.getString("releasedate"));
            fileWriter.write("\t");
            fileWriter.write(jsonObject.getString("certificateno") == null ? "" : jsonObject.getString("certificateno"));
            fileWriter.write("\t");
            fileWriter.write("\n");


        }
        fileWriter.flush();
        fileWriter.close();
    }

    public static String doGetData(String url, int pn, int ps) {
        String replace = url.replace("{{pn}}", String.valueOf(pn)).replace("{{ps}}", String.valueOf(ps));
        System.out.println(replace);
        String result = HttpClientUtil.get(replace);
        return result;
    }



    static class CompanyForkJoinTask extends RecursiveTask<JSONArray> {

        private int start;
        private int end;

        public CompanyForkJoinTask(int start, int end) {
            this.start = start;
            this.end = end;
        }

        @Override
        protected JSONArray compute() {
            if (end - start <= MAX) {
                JSONArray jsonArray = new JSONArray();
                JSONArray result = new JSONArray();
                for (int i = start; i < end; i++) {
                    String s = doGetData(url, i, 10);
                    JSONObject jsonObject = JSON.parseObject(s);
                    if (null == jsonObject || null == jsonObject.getInteger("flag")) {
                        System.out.println("get page size " + i + " return null");
                        continue;
                    }
                    if (!jsonObject.getInteger("flag").equals(0)) {
                        System.out.println("get page size " + i + " failure");
                        continue;
                    }
                    JSONObject data = jsonObject.getJSONObject("data");
                    if (null == data) {
                        System.out.println("get page size " + i + " return data is null");
                        continue;
                    }
                    Integer total = data.getInteger("total");
                    if (null == total || total < 1) {
                        System.out.println("get page size " + i + " is zero");
                        continue;
                    }

                    jsonArray.addAll(data.getJSONArray("list"));
                }


                for (int i = 0; i < jsonArray.size(); i ++) {
                    JSONObject jsonObject = jsonArray.getJSONObject(i);
                    String companyid = jsonObject.getString("companyid");
                    String u = url2.replace("{{cid}}", companyid);
                    String s = HttpClientUtil.get(u);
                    JSONObject jsonObject1 = JSON.parseObject(s);
                    if (null == jsonObject1) {
                        result.add(jsonObject);
                        continue;
                    }
                    JSONObject data = jsonObject1.getJSONObject("data");
                    if (null == data) {
                        result.add(jsonObject);
                        continue;
                    }
                    data.put("tel", jsonObject.getString("tel"));
                    result.add(data);
                }
                return result;
            } else {
                CompanyForkJoinTask t1 = new CompanyForkJoinTask(start, (start + end) / 2);
                CompanyForkJoinTask t2 = new CompanyForkJoinTask((start + end) / 2, end);
                invokeAll(t1, t2);
                JSONArray j1 = t1.join();
                JSONArray j2 = t2.join();
                j1.addAll(j2);
                return j1;
            }
        }
    }

    static class AreaForkJoinTask extends RecursiveTask<JSONArray> {

        private int start;
        private int end;

        public AreaForkJoinTask(int start, int end) {
            this.start = start;
            this.end = end;
        }

        @Override
        protected JSONArray compute() {
            if (end - start <= MAX) {
                JSONArray jsonArray = new JSONArray();
                JSONArray result = new JSONArray();
                for (int i = start; i < end; i++) {
                    String s = doGetData(area_url, i, 10);
                    JSONObject jsonObject = JSON.parseObject(s);
                    if (null == jsonObject || null == jsonObject.getInteger("flag")) {
                        System.out.println("get page size " + i + " return null");
                        continue;
                    }
                    if (!jsonObject.getInteger("flag").equals(0)) {
                        System.out.println("get page size " + i + " failure");
                        continue;
                    }
                    JSONObject data = jsonObject.getJSONObject("data");
                    if (null == data) {
                        System.out.println("get page size " + i + " return data is null");
                        continue;
                    }
                    Integer total = data.getInteger("total");
                    if (null == total || total < 1) {
                        System.out.println("get page size " + i + " is zero");
                        continue;
                    }

                    jsonArray.addAll(data.getJSONArray("list"));
                }


                for (int i = 0; i < jsonArray.size(); i ++) {
                    JSONObject jsonObject = jsonArray.getJSONObject(i);
                    String id = jsonObject.getString("keyid");
                    String u = area_detail_url.replace("{{id}}", id);
                    String s = HttpClientUtil.get(u);
                    JSONObject jsonObject1 = JSON.parseObject(s);
                    if (null == jsonObject1) {
                        result.add(jsonObject);
                        continue;
                    }
                    JSONObject data = jsonObject1.getJSONObject("data");
                    if (null == data) {
                        result.add(jsonObject);
                        continue;
                    }
//                    data.put("areamanager", jsonObject.getString("areamanager"));
//                    data.put("tel", jsonObject.getString("tel"));
                    result.add(data);
                }
                return result;
            } else {
                AreaForkJoinTask t1 = new AreaForkJoinTask(start, (start + end) / 2);
                AreaForkJoinTask t2 = new AreaForkJoinTask((start + end) / 2, end);
                invokeAll(t1, t2);
                JSONArray j1 = t1.join();
                JSONArray j2 = t2.join();
                j1.addAll(j2);
                return j1;
            }
        }
    }
}
